package com.tensquare.articlecrawler.processor;

import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.processor.PageProcessor;

/**
 * 文章处理类
 */
@Component
public class ArticleProcessor implements PageProcessor{
    /**
     * 页面解析
     * @param page
     */
    @Override
    public void process(Page page) {

        //1.发现新链接
        page.addTargetRequests(page.getHtml().regex("https://blog.csdn.net/[a-zA-Z0-9_]+/article/details/[0-9]{8}").all());

        //2.提前标题和内容
        //标题
        String title = page.getHtml().xpath("//*[@id=\"mainBox\"]/main/div[1]/div/div/div[1]/h1/text()").toString();
        //内容
        String content = page.getHtml().xpath("//*[@id=\"article_content\"]").toString();

        //3.把标题和内容传递给Pipeline
        if(title!=null && content!=null){
            page.putField("title",title);
            page.putField("content",content);
        }else{
            //跳过此页面请求
            page.setSkip(true);
        }
    }

    /**
     * 基本参数
     * @return
     */
    @Override
    public Site getSite() {
        return Site.me()
                .setSleepTime(100)
                .setTimeOut(1000)
                .setRetryTimes(3);
    }
}
